use rbpe::{BpeTokenizer, Tokenize};

fn main() {
    let train_data = std::fs::read_to_string("./data/train.txt").unwrap();
    let tokenizer = BpeTokenizer::train(&train_data, 100);
    
    let origin_texts = [
        "Learn about language model tokenization",
        "OpenAI's large language models process text using tokens, which are common sequences of characters found in a set of text. The models learn to understand the statistical relationships between these tokens, and excel at producing the next token in a sequence of tokens",
        "春风拂面花满枝，流水潺潺鸟语时。山川如画心自远，岁月静好梦相依。",
        "「今日も一日頑張りましょう！💪✨🌞」",
    ];

    for origin_test in origin_texts {
        let ids = Tokenize::encode(&tokenizer, origin_test);
        let text =  Tokenize::decode(&tokenizer, &ids);
        println!("{text}");
        assert_eq!(origin_test, text);
    }
}